{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fef25a3e-6b4b-4e3e-bc0c-df244156c169",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "normalFileList = os.listdir(\"item5-ss-data/normal/\")\n",
    "spamFileList = os.listdir(\"item5-ss-data/spam/\")\n",
    "print(\"正常邮件的文件列表\",normalFileList)\n",
    "print(\"垃圾邮件的文件列表\",spamFileList)\n",
    "stopList = []\n",
    "for line in open(\"item5-ss-data/stopwords.txt\", encoding='utf-8'):\n",
    "    stopList.append(line[:len(line) - 1])\n",
    "print(\"停用词文件内容:\",stopList)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54b69e58-3632-4935-9fc1-bb105104235c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from jieba import cut\n",
    "from re import sub\n",
    "def getWords(file,stopList):\n",
    "    wordsList=[]\n",
    "    for line in open(file,encoding='utf-8'):\n",
    "        line=line.strip()\n",
    "        line=sub(r'[.【】0-9、————，。！\\~*]','',line)\n",
    "        line=cut(line)\n",
    "        line=filter(lambda word:len(word)>1,line)\n",
    "        wordsList.extend(line)\n",
    "        words=[]\n",
    "        for i in wordsList:\n",
    "            if i not in stopList and i.strip()!='' and i!=None:\n",
    "                words.append(i)\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23e877c5-5d6f-41d4-a79c-8ba4427d84b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from itertools import chain\n",
    "allwords=[]\n",
    "for spamfile in spamFileList:\n",
    "    words=getWords(\"item5-ss-data/spam/\"+spamfile,stopList)\n",
    "    allwords.append(words)\n",
    "for normalfile in normalFileList:\n",
    "    words=getWords(\"item5-ss-data/normal/\"+normalfile,stopList)\n",
    "    allwords.append(words)\n",
    "print(\"训练集中所有的有效词语列表:\")\n",
    "print(allwords)\n",
    "frep=Counter(chain(*allwords))\n",
    "topTen=frep.most_common(10)\n",
    "topWords=[w[0] for w in topTen]\n",
    "print(\"训练集中出现频次最高的前10个词语:\")\n",
    "print(topWords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b966f059-072c-4a04-b1ff-694af8b54a5f",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'getWords' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[21], line 5\u001b[0m\n\u001b[0;32m      3\u001b[0m allwords\u001b[38;5;241m=\u001b[39m[]\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m spamfile \u001b[38;5;129;01min\u001b[39;00m spamFileList:\n\u001b[1;32m----> 5\u001b[0m     words\u001b[38;5;241m=\u001b[39mgetWords(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mitem5-ss-data/spam/\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m+\u001b[39mspamfile,stopList)\n\u001b[0;32m      6\u001b[0m     allwords\u001b[38;5;241m.\u001b[39mappend(words)\n\u001b[0;32m      7\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m normalfile \u001b[38;5;129;01min\u001b[39;00m normalFileList:\n",
      "\u001b[1;31mNameError\u001b[0m: name 'getWords' is not defined"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "from itertools import chain\n",
    "allwords=[]\n",
    "for spamfile in spamFileList:\n",
    "    words=getWords(\"item5-ss-data/spam/\"+spamfile,stopList)\n",
    "    allwords.append(words)\n",
    "for normalfile in normalFileList:\n",
    "    words=getWords(\"item5-ss-data/normal/\"+normalfile,stopList)\n",
    "    allwords.append(words)\n",
    "print(\"训练集中所有的有效词语列表:\")\n",
    "print(allwords)\n",
    "frep=Counter(chain(*allwords))\n",
    "topTen=frep.most_common(10)\n",
    "topWords=[w[0] for w in topTen]\n",
    "print(\"训练集中出现频次最高的前10个词语:\")\n",
    "print(topWords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3cabfd77-eee2-4985-8655-974f7dce2dc1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10个高频词语在每封邮件中出现的次数:\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "vector=[]\n",
    "for words in allwords:\n",
    "    temp=list(map(lambda x:words.count(x),topWords))\n",
    "    vector.append(temp)\n",
    "vector=np.array(vector)\n",
    "print(\"10个高频词语在每封邮件中出现的次数:\")\n",
    "print(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3c645603-3709-476c-ab6d-0de2328d6b1a",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Expected 2D array, got 1D array instead:\narray=[].\nReshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[25], line 5\u001b[0m\n\u001b[0;32m      3\u001b[0m x,y\u001b[38;5;241m=\u001b[39mvector,target\n\u001b[0;32m      4\u001b[0m model\u001b[38;5;241m=\u001b[39mMultinomialNB()\n\u001b[1;32m----> 5\u001b[0m model\u001b[38;5;241m.\u001b[39mfit(x,y)\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\base.py:1474\u001b[0m, in \u001b[0;36m_fit_context.<locals>.decorator.<locals>.wrapper\u001b[1;34m(estimator, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1467\u001b[0m     estimator\u001b[38;5;241m.\u001b[39m_validate_params()\n\u001b[0;32m   1469\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m config_context(\n\u001b[0;32m   1470\u001b[0m     skip_parameter_validation\u001b[38;5;241m=\u001b[39m(\n\u001b[0;32m   1471\u001b[0m         prefer_skip_nested_validation \u001b[38;5;129;01mor\u001b[39;00m global_skip_validation\n\u001b[0;32m   1472\u001b[0m     )\n\u001b[0;32m   1473\u001b[0m ):\n\u001b[1;32m-> 1474\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m fit_method(estimator, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\naive_bayes.py:732\u001b[0m, in \u001b[0;36m_BaseDiscreteNB.fit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    711\u001b[0m \u001b[38;5;129m@_fit_context\u001b[39m(prefer_skip_nested_validation\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m    712\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m, X, y, sample_weight\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m    713\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Fit Naive Bayes classifier according to X, y.\u001b[39;00m\n\u001b[0;32m    714\u001b[0m \n\u001b[0;32m    715\u001b[0m \u001b[38;5;124;03m    Parameters\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    730\u001b[0m \u001b[38;5;124;03m        Returns the instance itself.\u001b[39;00m\n\u001b[0;32m    731\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 732\u001b[0m     X, y \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_X_y(X, y)\n\u001b[0;32m    733\u001b[0m     _, n_features \u001b[38;5;241m=\u001b[39m X\u001b[38;5;241m.\u001b[39mshape\n\u001b[0;32m    735\u001b[0m     labelbin \u001b[38;5;241m=\u001b[39m LabelBinarizer()\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\naive_bayes.py:578\u001b[0m, in \u001b[0;36m_BaseDiscreteNB._check_X_y\u001b[1;34m(self, X, y, reset)\u001b[0m\n\u001b[0;32m    576\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_check_X_y\u001b[39m(\u001b[38;5;28mself\u001b[39m, X, y, reset\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[0;32m    577\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Validate X and y in fit methods.\"\"\"\u001b[39;00m\n\u001b[1;32m--> 578\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_data(X, y, accept_sparse\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcsr\u001b[39m\u001b[38;5;124m\"\u001b[39m, reset\u001b[38;5;241m=\u001b[39mreset)\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\base.py:650\u001b[0m, in \u001b[0;36mBaseEstimator._validate_data\u001b[1;34m(self, X, y, reset, validate_separately, cast_to_ndarray, **check_params)\u001b[0m\n\u001b[0;32m    648\u001b[0m         y \u001b[38;5;241m=\u001b[39m check_array(y, input_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124my\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mcheck_y_params)\n\u001b[0;32m    649\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 650\u001b[0m         X, y \u001b[38;5;241m=\u001b[39m check_X_y(X, y, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mcheck_params)\n\u001b[0;32m    651\u001b[0m     out \u001b[38;5;241m=\u001b[39m X, y\n\u001b[0;32m    653\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m no_val_X \u001b[38;5;129;01mand\u001b[39;00m check_params\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mensure_2d\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mTrue\u001b[39;00m):\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\utils\\validation.py:1263\u001b[0m, in \u001b[0;36mcheck_X_y\u001b[1;34m(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)\u001b[0m\n\u001b[0;32m   1258\u001b[0m         estimator_name \u001b[38;5;241m=\u001b[39m _check_estimator_name(estimator)\n\u001b[0;32m   1259\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m   1260\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mestimator_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m requires y to be passed, but the target y is None\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1261\u001b[0m     )\n\u001b[1;32m-> 1263\u001b[0m X \u001b[38;5;241m=\u001b[39m check_array(\n\u001b[0;32m   1264\u001b[0m     X,\n\u001b[0;32m   1265\u001b[0m     accept_sparse\u001b[38;5;241m=\u001b[39maccept_sparse,\n\u001b[0;32m   1266\u001b[0m     accept_large_sparse\u001b[38;5;241m=\u001b[39maccept_large_sparse,\n\u001b[0;32m   1267\u001b[0m     dtype\u001b[38;5;241m=\u001b[39mdtype,\n\u001b[0;32m   1268\u001b[0m     order\u001b[38;5;241m=\u001b[39morder,\n\u001b[0;32m   1269\u001b[0m     copy\u001b[38;5;241m=\u001b[39mcopy,\n\u001b[0;32m   1270\u001b[0m     force_all_finite\u001b[38;5;241m=\u001b[39mforce_all_finite,\n\u001b[0;32m   1271\u001b[0m     ensure_2d\u001b[38;5;241m=\u001b[39mensure_2d,\n\u001b[0;32m   1272\u001b[0m     allow_nd\u001b[38;5;241m=\u001b[39mallow_nd,\n\u001b[0;32m   1273\u001b[0m     ensure_min_samples\u001b[38;5;241m=\u001b[39mensure_min_samples,\n\u001b[0;32m   1274\u001b[0m     ensure_min_features\u001b[38;5;241m=\u001b[39mensure_min_features,\n\u001b[0;32m   1275\u001b[0m     estimator\u001b[38;5;241m=\u001b[39mestimator,\n\u001b[0;32m   1276\u001b[0m     input_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mX\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   1277\u001b[0m )\n\u001b[0;32m   1279\u001b[0m y \u001b[38;5;241m=\u001b[39m _check_y(y, multi_output\u001b[38;5;241m=\u001b[39mmulti_output, y_numeric\u001b[38;5;241m=\u001b[39my_numeric, estimator\u001b[38;5;241m=\u001b[39mestimator)\n\u001b[0;32m   1281\u001b[0m check_consistent_length(X, y)\n",
      "File \u001b[1;32mD:\\ProgramData\\anaconda3\\Lib\\site-packages\\sklearn\\utils\\validation.py:1035\u001b[0m, in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator, input_name)\u001b[0m\n\u001b[0;32m   1028\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   1029\u001b[0m             msg \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m   1030\u001b[0m                 \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpected 2D array, got 1D array instead:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124marray=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00marray\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1031\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mReshape your data either using array.reshape(-1, 1) if \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1032\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myour data has a single feature or array.reshape(1, -1) \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1033\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mif it contains a single sample.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1034\u001b[0m             )\n\u001b[1;32m-> 1035\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(msg)\n\u001b[0;32m   1037\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m dtype_numeric \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(array\u001b[38;5;241m.\u001b[39mdtype, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mkind\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m array\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39mkind \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUSV\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m   1038\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m   1039\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdtype=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnumeric\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m is not compatible with arrays of bytes/strings.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1040\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConvert your data to numeric values explicitly instead.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1041\u001b[0m     )\n",
      "\u001b[1;31mValueError\u001b[0m: Expected 2D array, got 1D array instead:\narray=[].\nReshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample."
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "target=np.array([1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0])\n",
    "x,y=vector,target\n",
    "model=MultinomialNB()\n",
    "model.fit(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9720a0fb-4696-4e3d-a626-d7b86342a2b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "test=os.listdir(\"item5-ss-data/test\")\n",
    "for testFile in test:\n",
    "    words=getWords(\"item5-ss-data/test/\"+testFile,stopList)\n",
    "    test_x=np.array(tuple(map(lambda x:words.count(x),topWords)))\n",
    "    result=model.predict(test_x.reshape(1,-1))\n",
    "    if result==1:\n",
    "        print('\"'+testFile+'\"'+\"是垃圾邮件\")\n",
    "    else:\n",
    "        print('\"'+testFile+'\"'+\"是正常邮件\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9b8c06d-b558-4dd8-89aa-f1dce2412119",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa0f659-68d5-476e-a456-7689ae4a4e1b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
